#!/usr/bin/env python
# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license.php 

"""
Class to provide preprocessing methods for Tweets, notably slang translation.
"""

import csv
import re
import getdict
import stemmer

class Preprocessor:
  """Performs preprocessing operations, such as slang dictionary lookups"""
  def __init__(self, newdict=False):
    # slang
    self.slangdict = {}
    if newdict:
      getdict.FetchDict()
    input = open(getdict.OUTPUT)
    lines = csv.reader(input)
    for line in lines:
      self.slangdict[line[0]] = line[1]
    # stopwords
    stopwordsfile = open('stopwords.txt')
    self.stopwords = stopwordsfile.next().strip().split(',')
    stopwordsfile.close()
    # porter stemmer
    self.mystemmer = stemmer.PorterStemmer()
  
  def SlangLookup(self, term):
    """Looks up term in slang dict, returns translation or false if not found"""
    if self.slangdict.has_key(term):
      return self.slangdict[term]
    return False
  
  def IsStopword(self, term):
    return term in self.stopwords
  
  def IsUrl(self, term):
    return bool(re.match('https?://', term))
  
  def IsWord(self, term):
    """Note - this matches numbers 0-9 too, which I think is okay."""
    return re.match('\w+', term)
    
  def Stem(self, string):
    return self.mystemmer.stem(string, 0, len(string) - 1)
  
  def Preprocess(self, string):
    string = string.lower()
    words = string.split(' ')
    newwords = []
    for word in words:
      if self.IsUrl(word):
        continue
      isWord = self.IsWord(word)
      if isWord:
        word = isWord.group(0)
      else:
        continue
      if self.IsStopword(word):
        continue
      slang = self.SlangLookup(word)
      if slang:
        # need special treatment as some slang translations are multiword
        slangwords = slang.split(' ')
        for slangword in slangwords:
          if not self.IsStopword(slangword):
            newwords.append(self.Stem(slangword))
      else:
        newwords.append(self.Stem(word))
    return newwords
